    def store_transition(self, s, a, r, s_):
        transition = []
        transition.append(s)
        transition.append(a)
        transition.append(r)
        transition.append(s_)
        transitionn = np.array(transition, dtype=object)
        if self.prioritized:    # prioritized replay
            #transition = np.hstack((s, [a, r], s_))
            self.memory.store(transitionn)    # have high priority for newly arrived transition
        else:       # random replay
            if not hasattr(self, 'memory_counter'):
                self.memory_counter = 0
            #transition = np.hstack((s, [a, r], s_))
            index = self.memory_counter % self.memory_size
            self.memory[index, :] = transitionn
            self.memory_counter += 1